library(foreign)
repDat <- read.dta("/Users/brucedesmarais/Dropbox/professional/Research/Submitted/Camp/Data/CandidateCluster/RepMergedPlus.dta")

# Start at col 48
library(glmnet)

y <- repDat$dhv
X <- as.matrix(repDat[,48:ncol(repDat)])

sig <- numeric(ncol(X))

for(i in 1:ncol(X)){
	est0 <- lm(y~as.numeric(X[,i])) 
	if(nrow(summary(est0)$coef) > 1){
	sig[i] <- summary(est0)$coef[2,4] < 0.05
	}
	if(i/100 == round(i/100)) print(i)
}

X <- X[,which(sig==1)]

colTime <- colnames(X)
colTime <- substr(colTime,10,13)
colTime <- as.numeric(colTime)

glmnVars <- NULL
grpNamesD <- NULL
grpNamesR <- NULL

for(t in seq(1990,2000,by=2)){

yt <- y[which(repDat$year==t)]
Xt <- X[which(repDat$year==t),which(colTime==t)]

glmn.est <- cv.glmnet(Xt,yt)

glmnOpt <- glmnet(Xt,yt,lambda=glmn.est$lambda.min)

betaMat <- as.matrix(glmnOpt$beta)

betaMat <- betaMat[betaMat[,1] !=0,]

commDat <- read.csv(paste("/users/brucedesmarais/Dropbox/professional/Research/Submitted/Camp/Data/Committees/committees",t,".csv",sep=""), header=F, stringsAsFactors=F)
colnames(commDat) <- c("Cycle","CmteID","PACShort","Affiliate","Ultorg","RecipID","RecipCode","FECCandID","Party","PrimCode","Source","Sensitive","Foreign","Active")

varT <- names(betaMat)
nums <- substr(varT,1,9)
part <- substr(varT,15,15)

nams <- commDat$PACShort[match(nums,commDat$CmteID)]

grpNamesD <- rbind(grpNamesD,cbind(nams[which(part=="D")],t,round(betaMat[which(part=="D")],dig=6)))
grpNamesR <- rbind(grpNamesR,cbind(nams[which(part=="R")],t,round(betaMat[which(part=="R")],dig=6)))

glmnVars <- c(glmnVars,names(betaMat))

print(t)

}

write.csv(grpNamesD,file="/Users/brucedesmarais/Dropbox/professional/Research/Submitted/Camp/Data/CandidateCluster/InfluentialPACsD.csv",row.names=F)
write.csv(grpNamesR,file="/Users/brucedesmarais/Dropbox/professional/Research/Submitted/Camp/Data/CandidateCluster/InfluentialPACsR.csv",row.names=F)


Xbest <- X[,glmnVars]

estBest <- lm(y~repDat$DemNotTriv+repDat$RepNotTriv+Xbest)

datWgroups <- data.frame(repDat[,1:47],Xbest)

write.dta(datWgroups,"/Users/brucedesmarais/Dropbox/professional/Research/Submitted/Camp/Data/CandidateCluster/RepMergedGrps.dta")
